{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba\n",
    "import jieba.posseg as pseg\n",
    "import jieba.analyse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 分析文本《球状闪电》:\n",
    "1. 分词\n",
    "2. 统计词频\n",
    "3. 统计词性频率\n",
    "4. 提取10个关键词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "分词词性标注完成\n"
     ]
    }
   ],
   "source": [
    "lines = []\n",
    "text = ''\n",
    "with open('data/qzsd.txt', 'r',) as f:\n",
    "    text = f.read()\n",
    "for l in text.split('\\n'):\n",
    "    l = l.strip()\n",
    "    if l != '':\n",
    "        lines.append(list(pseg.cut(l)))\n",
    "print('分词词性标注完成')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "分词结果保存成功\n"
     ]
    }
   ],
   "source": [
    "with open('data/qzsd/seg.txt', 'w') as segFile:\n",
    "    for l in lines:\n",
    "        for p in l:\n",
    "            p = list(p)\n",
    "            segFile.write(p[0] + '/' + p[1] + ' ')\n",
    "        segFile.write('\\n')\n",
    "    print('分词结果保存成功')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "词频结果保存成功\n"
     ]
    }
   ],
   "source": [
    "wc = {}\n",
    "with open('data/qzsd/wc.txt', 'w') as wcFile:\n",
    "    for l in lines:\n",
    "        for p in l:\n",
    "            p = list(p)\n",
    "            if wc.get(p[0]):\n",
    "                wc[p[0]] += 1\n",
    "            else:\n",
    "                wc[p[0]] = 1\n",
    "    for key in wc.keys():\n",
    "        wcFile.write(key + ' ' + str(wc[key]) + '\\n')\n",
    "    print('词频结果保存成功')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "词性频率结果保存成功\n"
     ]
    }
   ],
   "source": [
    "posc = {}\n",
    "with open('data/qzsd/posc.txt', 'w') as poscFile:\n",
    "    for l in lines:\n",
    "        for p in l:\n",
    "            p = list(p)\n",
    "            if posc.get(p[1]):\n",
    "                posc[p[1]] += 1\n",
    "            else:\n",
    "                posc[p[1]] = 1\n",
    "    for key in posc.keys():\n",
    "        poscFile.write(key + ' ' + str(posc[key]) + '\\n')\n",
    "    print('词性频率结果保存成功')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['闪电', '球状', '没有', '武器', '看到', '研究', '可能', '基地', '进行', '试验']\n"
     ]
    }
   ],
   "source": [
    "print(jieba.analyse.textrank(text, topK=10, withWeight=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 199801.txt文本中，统计：\n",
    "1. 出现次数最多的10个成语\n",
    "2. 词性种类最多的10个词\n",
    "3. 这10个词分别有哪些词性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['解放思想', 90]\n",
      "['艰苦奋斗', 71]\n",
      "['实事求是', 68]\n",
      "['全心全意', 37]\n",
      "['丰富多彩', 37]\n",
      "['坚定不移', 35]\n",
      "['欢聚一堂', 33]\n",
      "['不正之风', 30]\n",
      "['千方百计', 29]\n",
      "['坚持不懈', 28]\n",
      "['高', ['ad', 'an', 'a', 'j', 'nr', 'd', 'Ng', 'v']]\n",
      "['单', ['n', 'Ag', 'b', 'j', 'nr', 'd', 'Dg', 'Ng']]\n",
      "['和', ['n', 'c', 'Ag', 'nr', 'Ng', 'v', 'p']]\n",
      "['等', ['n', 'c', 'q', 'vn', 'a', 'v', 'u']]\n",
      "['近', ['ad', 't', 'a', 'j', 'd', 'Ng', 'v']]\n",
      "['来', ['y', 'm', 'vn', 'nr', 'f', 'v', 'u']]\n",
      "['独立', ['ad', 'an', 'vn', 'a', 'd', 'vd', 'v']]\n",
      "['多', ['m', 'ad', 'an', 'a', 'd', 'Ng', 'v']]\n",
      "['成', ['n', 'an', 'q', 'a', 'nr', 'j', 'v']]\n",
      "['白', ['b', 'a', 'j', 'nr', 'd', 'Ng', 'v']]\n"
     ]
    }
   ],
   "source": [
    "cys = {}\n",
    "posDict = {}\n",
    "with open('data/199801.txt', 'r', encoding='gbk') as rmrbFile:\n",
    "    for l in rmrbFile:\n",
    "        for pstr in l.split('  '):\n",
    "            p = pstr.split('/')\n",
    "            if len(p) > 1 and p[1] == 'i':\n",
    "                if cys.get(p[0]):\n",
    "                    cys[p[0]] += 1\n",
    "                else:\n",
    "                    cys[p[0]] = 1\n",
    "            if len(p) > 1:\n",
    "                if posDict.get(p[0]):\n",
    "                    posDict[p[0]].append(p[1])\n",
    "                    posDict[p[0]] = list(set(posDict[p[0]]))\n",
    "                else:\n",
    "                    posDict[p[0]] = [p[1]] # {'解放思想': ['i']}\n",
    "cyList = []\n",
    "for cy in cys.keys():\n",
    "    cyList.append([cy, cys[cy]])\n",
    "cyTop10 = sorted(cyList, key=lambda item: item[1], reverse=True)[:10]\n",
    "for cy in cyTop10:R\n",
    "    print(cy)\n",
    "\n",
    "posList = []\n",
    "for key in posDict.keys():\n",
    "    posList.append([key, posDict[key]])\n",
    "posTop10 = sorted(posList, key=lambda item: len(item[1]), reverse=True)[:10]\n",
    "for p in posTop10:\n",
    "    print(p)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
